{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 104)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../data/train_xy_all.csv')\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 159)"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_csv('../data/test_all.csv')\n",
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_x1 = pd.read_csv('../data/train_x.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_x2 = pd.read_csv('../data/train_xy.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# iv_feature = ['x_3', 'x_4', 'x_5', 'x_6', 'x_7', 'x_8', 'x_9', 'x_10', 'x_11', 'x_12', 'x_13', 'x_14', 'x_15', 'x_16', 'x_17', 'x_18', 'x_19', 'x_20', 'x_21', 'x_22', 'x_23', 'x_24', 'x_25', 'x_26', 'x_27', 'x_28', 'x_29', 'x_30', 'x_31', 'x_32', 'x_33', 'x_34', 'x_35', 'x_36', 'x_37', 'x_38', 'x_39', 'x_40', 'x_41', 'x_42', 'x_43', 'x_44', 'x_45', 'x_46', 'x_47', 'x_48', 'x_50', 'x_51', 'x_52', 'x_53', 'x_54', 'x_55', 'x_56', 'x_57', 'x_58', 'x_59', 'x_60', 'x_61', 'x_62', 'x_63', 'x_64', 'x_65', 'x_66', 'x_67', 'x_68', 'x_69', 'x_70', 'x_71', 'x_72', 'x_73', 'x_74', 'x_75', 'x_76', 'x_77', 'x_78', 'x_79', 'x_80', 'x_81', 'x_82', 'x_83', 'x_84', 'x_85', 'x_86', 'x_87', 'x_88', 'x_90', 'x_97', 'x_98', 'x_99', 'x_100', 'x_101', 'x_139', 'x_140', 'x_141', 'x_142', 'x_143', 'x_144', 'x_149', 'x_150', 'x_153', 'x_154', 'x_155', 'x_157']\n",
    "# x_test = test[iv_feature]\n",
    "# x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# iv_feature = ['x_3', 'x_4', 'x_5', 'x_6', 'x_7', 'x_8', 'x_9', 'x_10', 'x_11', 'x_12', 'x_13', 'x_14', 'x_15', 'x_16', 'x_17', 'x_18', 'x_19', 'x_20', 'x_21', 'x_22', 'x_23', 'x_24', 'x_25', 'x_26', 'x_27', 'x_28', 'x_29', 'x_30', 'x_31', 'x_32', 'x_33', 'x_34', 'x_35', 'x_36', 'x_37', 'x_38', 'x_39', 'x_40', 'x_41', 'x_42', 'x_43', 'x_44', 'x_45', 'x_46', 'x_47', 'x_48', 'x_50', 'x_51', 'x_52', 'x_53', 'x_54', 'x_55', 'x_56', 'x_57', 'x_58', 'x_59', 'x_60', 'x_61', 'x_62', 'x_63', 'x_64', 'x_65', 'x_66', 'x_67', 'x_68', 'x_69', 'x_70', 'x_71', 'x_72', 'x_73', 'x_74', 'x_75', 'x_76', 'x_77', 'x_78', 'x_79', 'x_80', 'x_81', 'x_82', 'x_83', 'x_84', 'x_85', 'x_86', 'x_87', 'x_88', 'x_90', 'x_97', 'x_98', 'x_99', 'x_100', 'x_101', 'x_139', 'x_140', 'x_141', 'x_142', 'x_143', 'x_144', 'x_149', 'x_150', 'x_153', 'x_154', 'x_155', 'x_157']\n",
    "# x_train = train[iv_feature]\n",
    "# x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_x2 = train_x2.drop(['y'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xxx = pd.concat([train_x2, train_x1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(35000, 159)"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = pd.concat([xxx, test])\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000,)"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train = train['y']\n",
    "Y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for i in range(3,80):\n",
    "#     col = 'x'+'_'+str(i)\n",
    "#     if col in xx.columns.values:\n",
    "#         dummies_df = pd.get_dummies(xx[col]).rename(columns=lambda x: col +'_'+ str(x))\n",
    "#         xx = pd.concat([xx, dummies_df], axis=1)\n",
    "# print(xx.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for i in range(82,93):\n",
    "#     col = 'x'+'_'+str(i)\n",
    "#     if col in xx.columns.values:\n",
    "#         dummies_df = pd.get_dummies(xx[col]).rename(columns=lambda x: col +'_'+ str(x))\n",
    "#         xx = pd.concat([xx, dummies_df], axis=1)\n",
    "# print(xx.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(35000, 366)\n"
     ]
    }
   ],
   "source": [
    "for i in range(96,158):\n",
    "    col = 'x'+'_'+str(i)\n",
    "    if col in x.columns.values:\n",
    "        dummies_df = pd.get_dummies(x[col]).rename(columns=lambda x: col +'_'+ str(x))\n",
    "        x = pd.concat([x, dummies_df], axis=1)\n",
    "print(x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# factor_col = [80, 81]\n",
    "# # x['x_1'] = pd.factorize(x['x_1'])[0]\n",
    "# xx['x_80'] = pd.factorize(xx['x_80'])[0]\n",
    "# xx['x_81'] = pd.factorize(xx['x_81'])[0]\n",
    "# # x['x_93'] = pd.factorize(x['x_93'])[0]\n",
    "# # x['x_95'] = pd.factorize(x['x_95'])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cust_id</th>\n",
       "      <th>cust_group</th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>x_8</th>\n",
       "      <th>...</th>\n",
       "      <th>x_156_1</th>\n",
       "      <th>x_156_2</th>\n",
       "      <th>x_156_3</th>\n",
       "      <th>x_157_-99</th>\n",
       "      <th>x_157_1</th>\n",
       "      <th>x_157_2</th>\n",
       "      <th>x_157_3</th>\n",
       "      <th>x_157_4</th>\n",
       "      <th>x_157_10</th>\n",
       "      <th>x_157_11</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110000</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.354167</td>\n",
       "      <td>0.604988</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110001</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.012058</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>110002</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.565979</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>110003</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.316209</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>110004</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.208333</td>\n",
       "      <td>0.008061</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 366 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cust_id cust_group       x_1       x_2  x_3  x_4  x_5  x_6  x_7  x_8  \\\n",
       "0   110000    group_3  0.354167  0.604988  -99  -99  -99  -99  -99  -99   \n",
       "1   110001    group_3  0.125000  0.012058  -99  -99  -99  -99  -99  -99   \n",
       "2   110002    group_3  0.333333  0.565979    0    0    0    0    0    0   \n",
       "3   110003    group_3  0.208333  0.316209    0    0    0    0    1    1   \n",
       "4   110004    group_3  0.208333  0.008061  -99  -99  -99  -99  -99  -99   \n",
       "\n",
       "     ...     x_156_1  x_156_2  x_156_3  x_157_-99  x_157_1  x_157_2  x_157_3  \\\n",
       "0    ...           0        0        1          1        0        0        0   \n",
       "1    ...           0        1        0          0        0        1        0   \n",
       "2    ...           0        1        0          0        0        1        0   \n",
       "3    ...           0        1        0          0        0        0        0   \n",
       "4    ...           0        1        0          0        1        0        0   \n",
       "\n",
       "   x_157_4  x_157_10  x_157_11  \n",
       "0        0         0         0  \n",
       "1        0         0         0  \n",
       "2        0         0         0  \n",
       "3        1         0         0  \n",
       "4        0         0         0  \n",
       "\n",
       "[5 rows x 366 columns]"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = x.drop(['cust_id','cust_group'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25000, 364)\n",
      "(10000, 364)\n"
     ]
    }
   ],
   "source": [
    "train_X = x[0:25000]\n",
    "test_X = x[25000:35000]\n",
    "print(train_X.shape)\n",
    "print(test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import metrics  #accuracy_score,recall_score,f1_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.utils.multiclass import unique_labels\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn import metrics\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.semi_supervised import label_propagation\n",
    "from sklearn import model_selection\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, KFold\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn import linear_model\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, BatchNormalization, Dropout, Reshape, Flatten, MaxPool2D\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D, Conv1D, MaxPooling1D\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_top_n_features(train_X, train_Y, top_n_features):\n",
    "\n",
    "    # random forest\n",
    "    rf_est = RandomForestClassifier(random_state=0)\n",
    "    rf_param_grid = {'n_estimators': [200], 'min_samples_split': [2], 'max_depth': [10]}\n",
    "    rf_grid = model_selection.GridSearchCV(rf_est, rf_param_grid, n_jobs=25, cv=10, verbose=1)\n",
    "    rf_grid.fit(train_X, train_Y)\n",
    "    print('Top N Features Best RF Params:' + str(rf_grid.best_params_))\n",
    "    print('Top N Features Best RF Score:' + str(rf_grid.best_score_))\n",
    "    print('Top N Features RF Train Score:' + str(rf_grid.score(train_X, train_Y)))\n",
    "    feature_imp_sorted_rf = pd.DataFrame({'feature': list(train_X),\n",
    "                                          'importance': rf_grid.best_estimator_.feature_importances_}).sort_values('importance', ascending=False)\n",
    "    features_top_n_rf = feature_imp_sorted_rf.head(top_n_features)['feature']\n",
    "    print('Sample 10 Features from RF Classifier')\n",
    "    print(str(features_top_n_rf[:10]))\n",
    "\n",
    "#     # AdaBoost\n",
    "#     ada_est =AdaBoostClassifier(random_state=0)\n",
    "#     ada_param_grid = {'n_estimators': [200], 'learning_rate': [0.01, 0.1]}\n",
    "#     ada_grid = model_selection.GridSearchCV(ada_est, ada_param_grid, n_jobs=25, cv=10, verbose=1)\n",
    "#     ada_grid.fit(train_X, train_Y)\n",
    "#     print('Top N Features Best Ada Params:' + str(ada_grid.best_params_))\n",
    "#     print('Top N Features Best Ada Score:' + str(ada_grid.best_score_))\n",
    "#     print('Top N Features Ada Train Score:' + str(ada_grid.score(train_X, train_Y)))\n",
    "#     feature_imp_sorted_ada = pd.DataFrame({'feature': list(train_X),\n",
    "#                                            'importance': ada_grid.best_estimator_.feature_importances_}).sort_values('importance', ascending=False)\n",
    "#     features_top_n_ada = feature_imp_sorted_ada.head(top_n_features)['feature']\n",
    "#     print('Sample 10 Feature from Ada Classifier:')\n",
    "#     print(str(features_top_n_ada[:10]))\n",
    "\n",
    "    # ExtraTree\n",
    "    et_est = ExtraTreesClassifier(random_state=0)\n",
    "    et_param_grid = {'n_estimators': [200], 'min_samples_split': [3], 'max_depth': [10]}\n",
    "    et_grid = model_selection.GridSearchCV(et_est, et_param_grid, n_jobs=25, cv=10, verbose=1)\n",
    "    et_grid.fit(train_X, train_Y)\n",
    "    print('Top N Features Best ET Params:' + str(et_grid.best_params_))\n",
    "    print('Top N Features Best ET Score:' + str(et_grid.best_score_))\n",
    "    print('Top N Features ET Train Score:' + str(et_grid.score(train_X, train_Y)))\n",
    "    feature_imp_sorted_et = pd.DataFrame({'feature': list(train_X),\n",
    "                                          'importance': et_grid.best_estimator_.feature_importances_}).sort_values('importance', ascending=False)\n",
    "    features_top_n_et = feature_imp_sorted_et.head(top_n_features)['feature']\n",
    "    print('Sample 10 Features from ET Classifier:')\n",
    "    print(str(features_top_n_et[:10]))\n",
    "\n",
    "#     # GradientBoosting\n",
    "#     gb_est =GradientBoostingClassifier(random_state=0)\n",
    "#     gb_param_grid = {'n_estimators': [200], 'learning_rate': [0.01], 'max_depth': [10]}\n",
    "#     gb_grid = model_selection.GridSearchCV(gb_est, gb_param_grid, n_jobs=25, cv=10, verbose=1)\n",
    "#     gb_grid.fit(train_X, train_Y)\n",
    "#     print('Top N Features Best GB Params:' + str(gb_grid.best_params_))\n",
    "#     print('Top N Features Best GB Score:' + str(gb_grid.best_score_))\n",
    "#     print('Top N Features GB Train Score:' + str(gb_grid.score(train_X, train_Y)))\n",
    "#     feature_imp_sorted_gb = pd.DataFrame({'feature': list(train_X),\n",
    "#                                            'importance': gb_grid.best_estimator_.feature_importances_}).sort_values('importance', ascending=False)\n",
    "#     features_top_n_gb = feature_imp_sorted_gb.head(top_n_features)['feature']\n",
    "#     print('Sample 10 Feature from GB Classifier:')\n",
    "#     print(str(features_top_n_gb[:10]))\n",
    "\n",
    "    # DecisionTree\n",
    "    dt_est = DecisionTreeClassifier(random_state=0)\n",
    "    dt_param_grid = {'min_samples_split': [2, 4], 'max_depth': [10]}\n",
    "    dt_grid = model_selection.GridSearchCV(dt_est, dt_param_grid, n_jobs=25, cv=10, verbose=1)\n",
    "    dt_grid.fit(train_X, train_Y)\n",
    "    print('Top N Features Best DT Params:' + str(dt_grid.best_params_))\n",
    "    print('Top N Features Best DT Score:' + str(dt_grid.best_score_))\n",
    "    print('Top N Features DT Train Score:' + str(dt_grid.score(train_X, train_Y)))\n",
    "    feature_imp_sorted_dt = pd.DataFrame({'feature': list(train_X),\n",
    "                                          'importance': dt_grid.best_estimator_.feature_importances_}).sort_values('importance', ascending=False)\n",
    "    features_top_n_dt = feature_imp_sorted_dt.head(top_n_features)['feature']\n",
    "    print('Sample 10 Features from DT Classifier:')\n",
    "    print(str(features_top_n_dt[:10]))\n",
    "\n",
    "    # merge the three models\n",
    "    features_top_n = pd.concat([features_top_n_rf, features_top_n_et, features_top_n_dt], \n",
    "                               ignore_index=True).drop_duplicates()\n",
    "\n",
    "    features_importance = pd.concat([feature_imp_sorted_rf, feature_imp_sorted_et, feature_imp_sorted_dt],ignore_index=True)\n",
    "\n",
    "    return features_top_n , features_importance\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 1 candidates, totalling 10 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=25)]: Done   5 out of  10 | elapsed:   34.1s remaining:   34.1s\n",
      "[Parallel(n_jobs=25)]: Done  10 out of  10 | elapsed:   43.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top N Features Best RF Params:{'min_samples_split': 2, 'max_depth': 10, 'n_estimators': 200}\n",
      "Top N Features Best RF Score:0.93032\n",
      "Top N Features RF Train Score:0.9516\n",
      "Sample 10 Features from RF Classifier\n",
      "76          x_80\n",
      "11          x_14\n",
      "49          x_53\n",
      "58          x_62\n",
      "57          x_61\n",
      "35          x_38\n",
      "151    x_153_-99\n",
      "17          x_20\n",
      "63          x_67\n",
      "99         x_153\n",
      "Name: feature, dtype: object\n",
      "Fitting 10 folds for each of 1 candidates, totalling 10 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=25)]: Done   5 out of  10 | elapsed:   35.9s remaining:   35.9s\n",
      "[Parallel(n_jobs=25)]: Done  10 out of  10 | elapsed:   44.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top N Features Best ET Params:{'min_samples_split': 3, 'max_depth': 10, 'n_estimators': 200}\n",
      "Top N Features Best ET Score:0.92272\n",
      "Top N Features ET Train Score:0.93568\n",
      "Sample 10 Features from ET Classifier:\n",
      "113       x_99_2\n",
      "99         x_153\n",
      "151    x_153_-99\n",
      "163    x_157_-99\n",
      "102        x_157\n",
      "152      x_153_1\n",
      "116      x_100_2\n",
      "104       x_97_1\n",
      "5            x_8\n",
      "17          x_20\n",
      "Name: feature, dtype: object\n",
      "Fitting 10 folds for each of 2 candidates, totalling 20 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=25)]: Done  13 out of  20 | elapsed:   21.1s remaining:   11.3s\n",
      "[Parallel(n_jobs=25)]: Done  20 out of  20 | elapsed:   31.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top N Features Best DT Params:{'min_samples_split': 2, 'max_depth': 10}\n",
      "Top N Features Best DT Score:0.91796\n",
      "Top N Features DT Train Score:0.95952\n",
      "Sample 10 Features from DT Classifier:\n",
      "76          x_80\n",
      "16          x_19\n",
      "88          x_99\n",
      "11          x_14\n",
      "163    x_157_-99\n",
      "77          x_81\n",
      "26          x_29\n",
      "57          x_61\n",
      "151    x_153_-99\n",
      "47          x_51\n",
      "Name: feature, dtype: object\n"
     ]
    }
   ],
   "source": [
    "feature_to_pick = 100\n",
    "feature_top_n, feature_importance = get_top_n_features(train_X, Y_train, feature_to_pick)\n",
    "train_feature = pd.DataFrame(train_X[feature_top_n])\n",
    "test_feature = pd.DataFrame(test_X[feature_top_n])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25000, 124)\n",
      "(10000, 124)\n"
     ]
    }
   ],
   "source": [
    "print(train_feature.shape)\n",
    "print(test_feature.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cust_id</th>\n",
       "      <th>cust_group</th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>x_8</th>\n",
       "      <th>...</th>\n",
       "      <th>x_156_1</th>\n",
       "      <th>x_156_2</th>\n",
       "      <th>x_156_3</th>\n",
       "      <th>x_157_-99</th>\n",
       "      <th>x_157_1</th>\n",
       "      <th>x_157_2</th>\n",
       "      <th>x_157_3</th>\n",
       "      <th>x_157_4</th>\n",
       "      <th>x_157_10</th>\n",
       "      <th>x_157_11</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100000</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.659675</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100001</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.657454</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100002</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.604167</td>\n",
       "      <td>0.825764</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100003</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.312500</td>\n",
       "      <td>0.473441</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100004</td>\n",
       "      <td>group_3</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.344635</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>-99</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 366 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cust_id cust_group       x_1       x_2  x_3  x_4  x_5  x_6  x_7  x_8  \\\n",
       "0   100000    group_3  0.125000  0.659675    0    1    1    0    3    3   \n",
       "1   100001    group_3  0.250000  0.657454    0    0    0    0    0    0   \n",
       "2   100002    group_3  0.604167  0.825764    0    0    0    0    3    3   \n",
       "3   100003    group_3  0.312500  0.473441    0    0    0    0    0    0   \n",
       "4   100004    group_3  0.458333  0.344635  -99  -99  -99  -99  -99  -99   \n",
       "\n",
       "     ...     x_156_1  x_156_2  x_156_3  x_157_-99  x_157_1  x_157_2  x_157_3  \\\n",
       "0    ...           0        1        0          0        0        1        0   \n",
       "1    ...           0        1        0          1        0        0        0   \n",
       "2    ...           0        1        0          1        0        0        0   \n",
       "3    ...           0        1        0          0        0        1        0   \n",
       "4    ...           0        1        0          0        1        0        0   \n",
       "\n",
       "   x_157_4  x_157_10  x_157_11  \n",
       "0        0         0         0  \n",
       "1        0         0         0  \n",
       "2        0         0         0  \n",
       "3        0         0         0  \n",
       "4        0         0         0  \n",
       "\n",
       "[5 rows x 366 columns]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train,X_val,y_train,y_val= train_test_split(train_X,Y_train,test_size=0.2,random_state=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gbm = XGBClassifier( n_estimators= 200, max_depth= 5, min_child_weight= 2, gamma=0.9, subsample=0.8, \n",
    "                        colsample_bytree=0.8, objective= 'binary:logistic', nthread= -1, scale_pos_weight=1).fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6667197115464363\n"
     ]
    }
   ],
   "source": [
    "predictions = gbm.predict_proba(X_val)\n",
    "pre = predictions[:,1]\n",
    "val_auc = metrics.roc_auc_score(y_val,pre)#验证集上的auc值\n",
    "print(val_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000,)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = gbm.predict_proba(test_X)\n",
    "pred = preds[:,1]\n",
    "pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Submission = pd.DataFrame({'cust_id': test['cust_id'], 'pred_prob': pred})\n",
    "Submission.to_csv('../result/semi_xgb1.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
